#include "../../../include/cppjieba/Jieba.hpp"
#include "../../../include/tinyxml2/tinyxml2.h"
#include "../../../include/offline/CreateInvertIndexTable.hpp"

#include <regex>
#include <iostream>
#include <string>
#include <vector>
#include <map>
#include <unordered_map>
#include <sstream>

using namespace std;
using namespace tinyxml2;

const char* const DICT_PATH = "../include/cppjieba/dict/jieba.dict.utf8";
const char* const HMM_PATH = "../include/cppjieba/dict/hmm_model.utf8";
const char* const USER_DICT_PATH = "../include/cppjieba/dict/user.dict.utf8";
const char* const IDF_PATH = "../include/cppjieba/dict/idf.utf8";
const char* const STOP_WORD_PATH = "../include/cppjieba/dict/stop_words.utf8";


InvertIndex::InvertIndex() {}


void InvertIndex::getPageContent() {
    XMLDocument doc;
    doc.LoadFile("../data/dat/newRipePage.dat");
    if (doc.ErrorID()) {
        cerr << "Load file failed." << endl;
        return;
    }

    XMLElement *itemNode = doc.FirstChildElement("doc");
    while (itemNode) {
        string docid = itemNode->FirstChildElement("docid")->GetText();
        string content = itemNode->FirstChildElement("content")->GetText();

        // cast docid from string to int
        const char* p = docid.c_str();
        int id = 0;
        id = atoi(p);

        regex reg("<[^>]+>"); // 通用正则表达式
        content = regex_replace(content, reg, "");

        // map<int, string> pageContent;
        _pageContent.insert(pair<int, string>(id, content));

        itemNode = itemNode->NextSiblingElement("doc");
    }

    /*test code*/
    // cout << "map size = " << pageContent.size() << endl;
    // for (auto &i:pageContent) {
    //     cout << i.second << endl;
    // }
}

void InvertIndex::calculateIDF() {
    // map<int, map<string, double>> frequencyWithId; // 每个网页的词频, TF
    set<string> wordList; // 所有网页去停用词后的单词
    map<string, int> pageContainWord; // DF

    // parse content
    cppjieba::Jieba jieba(DICT_PATH,
        HMM_PATH,
        USER_DICT_PATH,
        IDF_PATH,
        STOP_WORD_PATH);
    vector<string> words;
    // vector<cppjieba::Word> jiebawords;
    // string s;

    // store stop words from file into a set
    set<string> stopWords;
    ifstream ifs("../include/cppjieba/dict/stop_words.utf8");
    if (!ifs) {
        cout << "open file failed." << endl;
        exit(1);
    }

    string word;
    while (getline(ifs, word)) {
        stopWords.insert(word);
    }

    map<int, string>::iterator it = _pageContent.begin();
    for (int id = 1; it != _pageContent.end(); it++) {
        string content = _pageContent[id];
        // split words
        jieba.CutForSearch(content, words);

        map<string, int> wordCount;
        map<string, double> wordFrequency;
        // check if the word is a stop word
        for (auto &i:words) {
            int totalWordsInPage = words.size();
            auto isStopWord = stopWords.find(i);
            for (auto it = stopWords.begin(); it != stopWords.end(); it++) {
                if (isStopWord == it) { // the word is a stop word
                    break;
                } else if (isStopWord == stopWords.end()) { // the word is not a stop word
                    wordList.insert(i);
                    ++wordCount[i];
                    wordFrequency[i] = wordCount[i] / (double) totalWordsInPage;
                    if (wordCount[i] == 1) { // first appear in this page
                        if (pageContainWord.count(i) == 1) {
                            ++pageContainWord[i];
                        } else {
                            pageContainWord.insert(pair<string, int>(i, 1));
                        }
                    } 
                    break;
                }
            }
        }
        _frequencyWithId.insert(pair<int, map<string, double>>(id, wordFrequency));       
        id++;
    }

    /*test code*/
    // for (auto &e:frequencyWithId) {
    //     cout << e.first << " " << e.second << endl;
    // }
    // for (auto &a:wordList) {
    //     cout << a << endl;
    // }
    // for (auto &e:pageContainWord) {
    //     cout << e << endl;
    // }


    // calculate IDF
    // double idf = 0;
    double n = _pageContent.size(); // total number of pages
    // map<string, double> wordIDF;

    for (auto iter = pageContainWord.begin(); iter != pageContainWord.end(); iter++) {
        double idf = 0.00;
        double tmp = iter->second;       
        idf = log10(n / tmp + 1);

        _wordIDF.insert(pair<string, double>(iter->first, idf));      
    }

    /*test code*/
    // for (auto &i:wordIDF) {
    //     cout << i << endl;
    // }

    ifs.close();
}

void InvertIndex::calculateWeight() {
    // calculate initial weight
    map<int, map<string, double>> initialWeight;
    for (auto it = _frequencyWithId.begin(); it != _frequencyWithId.end(); it++) { // docid
        auto tmp = it->second.begin(); // word
        map<string, double> tempMap;
        for (; tmp != it->second.end(); tmp++) { // TF
            double weight = 0.00;
            weight = tmp->second * _wordIDF[tmp->first];
            tempMap[tmp->first] = weight;
        }
        initialWeight.insert(pair<int, map<string, double>>(it->first, tempMap));
    }

    // normalization
    // map<int, map<string, double>> normalizedWeight;
    for (auto it = initialWeight.begin(); it != initialWeight.end(); it++) { // docid
        auto tmp = it->second.begin(); // word       
        double sum = 0.00;
        for (; tmp != it->second.end(); tmp++) {
            sum += pow(tmp->second, 2);
        }
        double denominator = 0.00;
        denominator = sqrt(sum);
        map<string, double> tempMap;
        // cout << "sum = " << sum << " denominator = " << denominator << endl;
        for (auto i = it->second.begin(); i != it->second.end(); i++) {
            double normalWeight = 0.00;
            normalWeight = i->second / denominator;
            tempMap[i->first] = normalWeight;  
        }
        _normalizedWeight.insert(pair<int, map<string, double>>(it->first, tempMap));
    }

    /*test code*/
    // for (auto &e:initialWeight) {
    //     cout << e << endl;
    // }
    // for (auto &t:normalizedWeight) {
    //     cout << t << endl;
    // }
}

void InvertIndex::generateInvertIndexTable() {
    // unordered_map<string, vector<pair<int, double>>> invertIndexTable;

    // get all words
    set<string> wordList;
    for (auto it = _normalizedWeight.begin(); it != _normalizedWeight.end(); it++) {
        for (auto itMap = it->second.begin(); itMap != it->second.end(); itMap++) {
            wordList.insert(itMap->first);
        }
    }

    for (auto &word:wordList) {
        vector<pair<int, double>> weightWithId;
        for (auto it = _normalizedWeight.begin(); it != _normalizedWeight.end(); it++) {
            auto exist = it->second.find(word);
            if (it->second.find(word) != it->second.end()) {
                weightWithId.push_back(pair<int, double>(it->first, exist->second));
            }           
        }
        _invertIndexTable.insert(pair<string, vector<pair<int, double>>>(word, weightWithId));
    }

    /*test code*/
    // for (auto &t:invertIndexTable) {
    //     cout << t << endl;
    // }

    // output to the invertIndex.dat
    ofstream ofs("../data/dat/invertIndex.dat");
    if (!ofs) {
        cerr << "open newTest.dat failed" << endl;
        return;
    }

    for (auto &elem:_invertIndexTable) {
        ofs << elem.first << " ";
        // cout << elem.first << " ";
        for (auto it = elem.second.begin(); it != elem.second.end(); it++) {
            // it->first docid  it->second w'
            // cout << it->first << " " << it->second << " ";
            ofs << it->first << " " << it->second << " ";
        }
        ofs << '\n';
        // cout << endl;
    }
}


int main() {
    // 生成倒排索引库
    cout << "=== Start generating invertIndex.dat ===" << endl << endl;
    InvertIndex invertIndex;
    invertIndex.getPageContent();
    invertIndex.calculateIDF();
    invertIndex.calculateWeight();
    invertIndex.generateInvertIndexTable();
    cout << "=== invertIndex.dat was successfully generated ===" << endl << endl;

    return 0;
}